##Construct a submission
require(parallel)
args<-commandArgs(TRUE)
perf.info<-read.csv("perf_info.csv", stringsAsFactors=F, header=T)
filename<-paste("/vol/test/Test",args[1],".csv",sep="")

df_sample<-read.csv("~/facebook/my_test.csv", stringsAsFactors = F, header=F, col.names=c("Id","Title","Body"))

v.text<-tolower(df_sample$Body)
v.title<-tolower(df_sample$Title)

f<-function(i)
{
  print(i)

  #grep for tag in body of the post
  re<-perf.info$tag.map.regexp[i]
  ind.pred.body <- grep(re, v.text, perl=T)

  #grep for tag in title of the post
  ind.pred.title <- grep(re, v.title, perl=T, useBytes=T)

  ind.pred.both <- intersect(ind.pred.body, ind.pred.title)
  ind.pred.body.only <- setdiff(ind.pred.body, ind.pred.title)
  ind.pred.title.only <- setdiff(ind.pred.title, ind.pred.body)

  list(res.tagnum=i,tag.name=perf.info$tag[i], ind.pred.both=ind.pred.both, ind.pred.body.only=ind.pred.body.only,
       ind.pred.title.only=ind.pred.title.only, spec1=perf.info$spec1[i], spec2=perf.info$spec2[i], spec3=perf.info$spec3[i])
}

num.tags<-nrow(perf.info)
b<-mclapply(1:num.tags, f, mc.cores=4)

spec1 <- sapply(1:length(b), function(i) b[[i]]$spec1)
spec2 <- sapply(1:length(b), function(i) b[[i]]$spec2)
spec3 <- sapply(1:length(b), function(i) b[[i]]$spec3)


spec<-rep("", nrow(df_sample))
predictions <- rep("", nrow(df_sample))

for(i in 1:length(b))
{

  ind <- b[[i]]$ind.pred.both
  predictions[ind] <- paste(predictions[ind],perf.info$tag[i])
  spec[ind]<-paste(spec[ind],spec1[i])

  ind <- b[[i]]$ind.pred.body.only
  predictions[ind] <- paste(predictions[ind],perf.info$tag[i])
  spec[ind]<-paste(spec[ind],spec2[i])

  ind <- b[[i]]$ind.pred.title.only
  predictions[ind] <- paste(predictions[ind],perf.info$tag[i])
  spec[ind]<-paste(spec[ind],spec3[i])
}

spec<-substring(spec, 2)
predictions<-substring(predictions, 2)

num.to.keep<-function(p)
{
  a<-vector("list",length(p))
  a<-lapply(1:length(a), function(i) a[[i]]<-c(0,1))
  outcomes<-expand.grid(a)
  prob.outcome<-sapply(1:nrow(outcomes),function(i)prod(p[unlist(outcomes[i,]>0)])*prod(1-p[unlist(outcomes[i,]==0)]))
  Ef<-rep(NA, length(p))
  for(j in 1:length(p))
  {
    a.try<-rep(0, length(p))
    a.try[1:j]<-1
    tp<-as.matrix(outcomes)%*%a.try
    prec<-tp/j
    rec<-tp/apply(outcomes, 1, sum)
    f<-2*prec*rec/(prec+rec)
    f[is.na(f)]<-0
    Ef[j]<-sum(f*prob.outcome)
    #cat(j, Ef, "\n")
  }
  if(max(Ef)<.1) 0
  else which.max(Ef)
}

##Now predictions must be ordered by specificity and only the most powerful ones retained

g <- function(i)
{
  print(i)
  if(predictions[i]=="") return("")
  v_pred <- unlist(strsplit(predictions[i]," "))
  v_spec <- as.numeric(unlist(strsplit(spec[i]," ")))
  v_pred <- v_pred[rev(order(v_spec))]
  v_spec <- v_spec[rev(order(v_spec))]
  if (length(v_pred)>5)
  {
    v_pred <- v_pred[1:5]
    v_spec <- v_spec[1:5]
  }
  print(v_spec)
  n <- num.to.keep(v_spec)
  cat(length(v_pred), " predictions, keeping ", n, "\n")
  if (n==0) ""  else   paste(v_pred[1:n],collapse=" ")
}

ans <- mclapply(1:length(predictions), g, mc.cores=4)

predictions<-unlist(ans)
predictions[predictions==""] <- "c# java javascript c++"

outfile<-paste("/vol/submission",args[1],".csv",sep="")
df_submission<-data.frame(Id=df_sample$Id, Tags=predictions)
write.csv(df_submission, outfile, row.names=FALSE)

 s = u'привет'
